package com.xiaotu.spider.project.pageprocessor.list;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.google.gson.Gson;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 SohuCommentListPageProcessor
 * @日期 2017年8月29日
 * @作者 王艳龙
 * @功能  搜狐爬取类--TV 短评
 */
@Component("SohuCommentListPageProcessor")
@Scope("prototype")
public class SohuCommentListPageProcessor extends AUrlPageProcessor
{
	
    private String fileName; //读取tvName文件路径
    private String SoHuComment; //搜狐短评
    
    @Override
    protected String[] getStartRequestUrls()
    {
        List<JSONObject> targetList = readFileColumn(fileName); //读取excel电影名
        List<String> linkList = new ArrayList<String>();
        for (JSONObject target : targetList)
        {
            linkList.add("http://so.tv.sohu.com/mts?box=1&wd="
				+ KeywordEncode.encode(target.getString("tvName")));
        }
        String[] urlArr = new String[linkList.size()];
        linkList.toArray(urlArr);
        return urlArr;

    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("fileName") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileName = map.get("fileName") + StringUtils.EMPTY;

        if (map.get("SoHuComment") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        SoHuComment = map.get("SoHuComment") + StringUtils.EMPTY;
    }

    @Override
    public String getPageDataType(Page page)
    {
    	String url = page.getUrl().toString();
		if (url.contains("http://so.tv.sohu.com/mts?box=1&wd="))
			return Constants.PageDataType.SEARCH_LIST;
		else if (url.contains("/comments"))
			return Constants.PageDataType.TV_COMMENT;
		else
			return Constants.PageDataType.SEARCH_LIST;
    }

    @Override
    public void process(Page page)
    {
    	Map<String, Object> tvMap = page.getRequest().getExtras();
		switch (this.getPageDataType(page))
		{
			case Constants.PageDataType.SEARCH_LIST:
				this.searchListProcess(page, tvMap);
				break;
			case Constants.PageDataType.TV_COMMENT:
				this.commentProcess(page, tvMap);
				break;
			default:
				break;
		}
    }

    private void searchListProcess(Page page, final Map<String, Object> tvMap)
	{
		Html html = page.getHtml();
		//网大名称
		String movieName =html.xpath("//div[@class='ssItem cfix']/div[@class='center']/div[@class='infoA cfix lan_infoA']/h2/a/@title").toString();
		if(StringUtils.isNotEmpty(movieName)){
			tvMap.put("movieName", movieName);
		}
		// 链接标签
		String linkUrl = html.xpath("//div[@class='ssItem cfix']/div[@class='center']/div[@class='infoA cfix lan_infoA']/h2/a").toString();
		if(StringUtils.isNotEmpty(linkUrl)){
			Html aHtml = new Html(linkUrl);
			String playId = aHtml.xpath("//a/@_s_a").get();
			//获取总页数
			String jsonStr = HttpUtils.httpGetJSONP("http://changyan.sohu.com/api/2/topic/load?client_id=cyqyBluaj&topic_url="
					+ aHtml.xpath("//a/@href").get()+ "&topic_source_id=vp" + playId+ "&page_size=30");
			jsonStr = jsonStr.substring(jsonStr.indexOf("{"));
			jsonStr = jsonStr.substring(0, jsonStr.lastIndexOf("}") + 1);
			JSONObject C_json = JSONObject.fromObject(jsonStr);
			String topicId =C_json.getString("topic_id");
			String totalPage = C_json.getString("total_page_no");
			int pageSize =Integer.valueOf(totalPage);
			
			String linkUrls = html.xpath("//div[@class='siteSeries cfix']/div[@class='btnBox']/a").toString();
			Html aHtmls = new Html(linkUrls);
			String sourceId = aHtmls.xpath("//a/@_s_v").get();
			for (int i = 1; i <= pageSize; i++)
			{
				PageUtils.addTargetRequest(page,
						"http://changyan.sohu.com/api/2/topic/comments?client_id=cyqyBluaj"
								+ "&page_no="+i+"&page_size=30"
								+ "&topic_source_id="+sourceId+""
								+ "&topic_id="+topicId+""
								+ "&order_by=time&callback=jQuery172035226931720576093_1503999790893&_=1504000076159",
								tvMap);// 添加评论链接
			}
		}
	}
	
	private void commentProcess(Page page, Map<String, Object> tvMap)
	{
		String jsonStr = page.getRawText();
		// format json
		jsonStr = jsonStr.substring(jsonStr.indexOf("{"));
		
		jsonStr = jsonStr.substring(0, jsonStr.lastIndexOf("}") + 1);
		
		JSONObject C_json = JSONObject.fromObject(jsonStr);
		
		if (!C_json.containsKey("comments"))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		JSONArray comments = C_json.getJSONArray("comments");
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
		for (int i = 0; i < comments.size(); i++)
		{
			JSONObject comment = comments.getJSONObject(i);
			
			if (!comment.containsKey("content"))
				continue;
			
			if (!comment.containsKey("create_time")
					|| StringUtils.isEmpty(comment.getString("create_time")))
				continue;
			
			if (!comment.containsKey("passport"))
				continue;
			Date data = new Date(Long.valueOf(comment.getString("create_time")));
			String pubtime =sdf.format(data);
			String line = StringUtils.join(new String[]{StringUtil.nullToString(page.getRequest().getExtra("type")),
					KeywordEncode.decode((String)tvMap.get("movieName")), 
					comment.getJSONObject("passport").getString("nickname").replace("'", "''"), 
					pubtime, 
					comment.getString("content").replace("'", "''")}, SepratorUtil
	                .SEP_PIPE);
	        System.out.println(line);
	        FileUtil.writeFile(SoHuComment, line + SepratorUtil.LINE_SEPRATOR);
		}
	}
    /**
     * 读取文件指定列
     *
     * @param fileName 文件名称
     * @return 行列表
     */
    public static List<JSONObject> readFileColumn(String fileName)
    {
        Map<String, Object> map = new HashMap<String, Object>();
        Workbook wb = null;
        String json;
        List<JSONObject> list = new ArrayList<JSONObject>();
        try
        {
            InputStream inputStream = new FileInputStream(fileName);
            if (fileName.endsWith("xls"))
            {
                wb = new HSSFWorkbook(inputStream); // 解析xls格式
            } else if (fileName.endsWith("xlsx"))
            {
                wb = new XSSFWorkbook(inputStream); // 解析xlsx格式
            }
            Sheet sheet = wb.getSheetAt(0); // 第一个工作表

            int firstRowNum = 1;
            int lastRowNum = sheet.getLastRowNum();

            Row row = null;
            Cell cell_a1 = null;
            for (int i = firstRowNum; i <= lastRowNum; i++)
            {
                row = sheet.getRow(i);          //取得第i行
                cell_a1 = row.getCell(0);      //取得i行的第一列
                if(row.getCell(0)!=null){
                	cell_a1.setCellType(Cell.CELL_TYPE_STRING);
                    String cellValue1 = cell_a1.getStringCellValue();
                    map.put("tvName", cellValue1);
               }
                json = new Gson().toJson(map);
                JSONObject jsStr = JSONObject.fromObject(json);
                list.add(jsStr);
            }

        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return list;
    }

    @Override
    protected String getSpiderType()
    {
        return "SohuCommentList";
    }
}
